{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.5571604,  0.       ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('MountainCarContinuous-v0', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "\n",
    "        reward = -1.0\n",
    "        if done:\n",
    "            reward = 100\n",
    "\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhdUlEQVR4nO3daWwTd/4/8Pc4PuIcdm6bEKekQBtS7pwmgXKkpCy7FKj2qPqg6q5a0YZVaatKjfQv1UorpdrVSrvdreiD1bZ9sCorVqUHgtCQQA7iHE5ICYGGwkITiO0AIXYOx9d8/g9o5oc52iSETGx/XtIIMvO1/fGEeTPznZnvCEREYIwxGSjkLoAxFrk4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjspEtgD744AMsWLAA0dHRKCwsRGtrq1ylMMZkIksA/ec//8Ebb7yBd999Fx0dHVixYgXKysowMDAgRzmMMZkIctyMWlhYiPz8fPzjH/8AAIiiCJPJhN///vd4++23Z7scxphMlLP9gV6vF+3t7aioqJDmKRQKlJaWwmKx3PM1Ho8HHo9H+lkURQwODiI5ORmCIDz0mhljU0NEGB4eRnp6OhSK+x9ozXoAXb9+HYFAAAaDIWi+wWDAt99+e8/XVFZW4g9/+MNslMcYm0F9fX3IyMi47/JZD6DpqKiowBtvvCH97HQ6kZmZib6+Puh0OhkrY4zdi8vlgslkQnx8/I+2m/UASklJQVRUFBwOR9B8h8MBo9F4z9doNBpoNJq75ut0Og4gxuawn+oimfWzYGq1Grm5uaipqZHmiaKImpoamM3m2S6HMSYjWQ7B3njjDbzwwgvIy8tDQUEB/vrXv2J0dBQvvviiHOUwxmQiSwD9+te/xrVr17B3717Y7XasXLkSVVVVd3VMM8bCmyzXAT0ol8sFvV4Pp9PJfUCMzUGT3UZD4iwYY+zh83qvIhC4CZXKhKioGADKh36dHQcQYwwAMDT0Ofr7/x+UyjRER2cjJmYFtNrliI5+AiqVAVFRekyct5qpYOIAYoz9QEQgMIRAYAgez3k4nV8CUEKhiIZabYJGswixsQXQ6coQG5s/I5/IAcQYuw8FlMoEKJXzoNU+Ie0RqVTpM/YJHECMMQCAz6dEdPRKaLVLfgibVdBosqBSpUOhiAagmPE+IQ4gxhgAoKMjFWvWfIGkpPkQhKhZ+UweEZExBgDw+RQQhLhZCx+AA4gxJiMOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbPheMMbYT7p94FQiCrop9UFuUOUAYowFISK4XC6MjIzAZrPh2rVrGB8fh9PphFarxcjICLRaLVQqFaKjo5GUlIS4uDjMnz8f8fHxUKvVk/4sDiDGIhwRwefzYXBwEEePHoUoinC73TAajUhNTcWiRYuQmJgIlUoV9JhlURQxPj6OwcFB2Gw2XLp0CU6nE7GxsUhMTJzUZ3MAMRaBiAiiKMJms+HUqVO4fv06oqOjkZubi8zMzKAHgf7YIZZer4fBYEB2drY0b2hoCD09PZOqgwOIsQjj8Xhw/vx5WK1WGI1GrF69GmlpaVAqpz8I/e2vS0xMRE5OzqRexwHEWASYOMzq6urCmTNnkJ2djV/+8peIi4uTtS4OIMbCXCAQQHd3N7q6urBkyRI899xzUKlUD/2RO5PBAcRYmCIiXL16FW1tbUhJScH27dsRExMzJ4JnAgcQY2HI6/WitbUVDocDGzduREJCwpwKngkcQIyFESJCf38/jh8/jry8PJjNZkRFzd4Yz1PFAcRYmAgEAujs7ITD4cC2bdsQHx8/J/d6bjfle8Hq6+vxi1/8Aunp6RAEAZ9//nnQciLC3r17MW/ePGi1WpSWluK7774LajM4OIjnn38eOp0OCQkJ+N3vfoeRkZEH+iKMRbLR0VEcPXoUgiCgrKwMOp1uzocPMI0AGh0dxYoVK/DBBx/cc/mf/vQnvP/++/jwww/R0tKC2NhYlJWVYXx8XGrz/PPPo7u7G9XV1Th06BDq6+vx8ssvT/9bMBahiAh9fX04duwYVq1ahVWrVs3pQ6670AMAQAcPHpR+FkWRjEYj/fnPf5bmDQ0NkUajoU8//ZSIiM6ePUsAqK2tTWpz5MgREgSBrl69OqnPdTqdBICcTueDlM9YSBNFkb799lv673//SyMjIySKotwlSSa7jc7ocByXLl2C3W5HaWmpNE+v16OwsBAWiwUAYLFYkJCQgLy8PKlNaWkpFAoFWlpa7vm+Ho8HLpcraGIskvn9flgsFly7dg3btm1DbGxsSBxy3WlGA8hutwMADAZD0HyDwSAts9vtSEtLC1quVCqRlJQktblTZWUl9Hq9NJlMppksm7GQ4vP50NDQAJVKheLiYqhUKrlLmraQGJCsoqICTqdTmvr6+uQuibFZR0QYHx9HXV0dMjMzkZeXF5J7Pbeb0QAyGo0AAIfDETTf4XBIy4xGIwYGBoKW+/1+DA4OSm3upNFooNPpgibGIgkRwePxoLq6Go8//jgeffTRkA8fYIYDKCsrC0ajETU1NdI8l8uFlpYWmM1mAIDZbMbQ0BDa29ulNrW1tRBFEYWFhTNZDmNhY3R0FLW1tSgqKkJGRkZYhA8wjQsRR0ZGcOHCBennS5cuobOzE0lJScjMzMSePXvwxz/+EYsXL0ZWVhbeeecdpKenY/v27QCAJUuW4Omnn8ZLL72EDz/8ED6fD7t378ZvfvMbpKenz9gXYywcEBFGRkZQU1ODdevWISkpSe6SZtZUT68dP36cANw1vfDCC0R069TgO++8QwaDgTQaDW3atIl6enqC3uPGjRv03HPPUVxcHOl0OnrxxRdpeHh40jXwaXgWKVwuFx05coRu3rw5p06z/5TJbqMC0W2jTYcIl8sFvV4Pp9PJ/UEsLBERhoaGcPLkSZSUlCAhIUHukqZksttoSJwFYyySEBHGxsZw8uRJmM1m6PV6uUt6aPhmVMbmmPHxcZw4cQJr1qwJvz6fO/AeEGNziMfjQU1NDfLy8ib9ZIlQxgHE2BwxcYXzypUrkZaWFjan2n8MBxBjc4DP50NNTQ0effRRzJ8/PyLCB+AAYkx2RITW1lYYjUZkZWVFTPgAHECMyYqI0NXVBbVajRUrVkRU+AAcQIzJhohw5swZ9Pf3Izc3N+LCB+AAYkw2NpsN/f392LhxY9Az1yNJZH5rxmQ2cUP2unXroFar5S5HNhxAjM0yt9uNw4cPw2w2Izo6Wu5yZBXSV0KH4G1sLMIFAgHU19fjySefREpKitzlyC6k94BcLheHEAsZoijCYrEgIyODh575QUgHUFNTE7xer9xlMPaTiAjnzp2D1+tFTk5ORJ7xupeQDqDs7GzU1NTA7/fLXQpj90VEcDgc6Ovrw9q1azl8bhPSAbRgwQKkpKSgs7OTD8XYnDUyMgKLxYJ169aF9BMsHoaQDiBBEJCbmwuXy4Xvv/+eQ4jNOT6fDydOnEB+fj60Wq3c5cw5IR1AABAVFYWSkhJYrVbcvHlT7nIYk4iiiObmZjz22GMRdYPpVIR8AAGASqXC+vXr0dDQAI/HI3c5jEm3WSgUCixevJjD5z7CIoAEQUBycjKWLVuGpqYmiKIod0ksghERrl27hkuXLqGwsDBib7OYjLBZM4IgICsrC7GxsTh16hT3BzHZuN1uNDc3Y+PGjVAqQ/pa34cubAIIuBVCq1evht1uh91u5xBis87v96O2tharV69GfHy83OXMeWEVQACgVCqxYcMGtLS0wO12y10OiyBEBKvVitTUVMyfP1/uckJC2AUQAGi1WuTn5+PYsWPw+Xxyl8MiABHhypUrGB4eRl5eHnc6T1JYBpAgCEhPT0d6ejr3B7FZ4XK50NrairVr1yIqKkruckJGWAYQcCuEVq1aheHhYfT29spdDgtjfr8fNTU1KCwshEajkbuckBK2AQTcukixuLgYVqsVg4ODvCfEZtxEv89jjz2GjIwMPvSaorAOIADQaDQwm82oq6tDIBCQuxwWZi5evIixsTHk5OTIXUpImlIAVVZWIj8/H/Hx8UhLS8P27dvR09MT1GZ8fBzl5eVITk5GXFwcnn32WTgcjqA2vb292Lp1K2JiYpCWloa33nrrod3RLggC5s2bh8WLF8NqtfJeEJsxQ0NDaG1tRVFREV9sOE1TWmt1dXUoLy9Hc3Mzqqur4fP5sHnzZoyOjkptXn/9dXz11Vc4cOAA6urq0N/fj507d0rLA4EAtm7dCq/Xi6amJnzyySf4+OOPsXfv3pn7VncQBAFPPPEERkZGcPHiRQ4h9sB8Ph8aGxvx1FNPISYmRu5yQhc9gIGBAQJAdXV1REQ0NDREKpWKDhw4ILU5d+4cASCLxUJERIcPHyaFQkF2u11qs2/fPtLpdOTxeCb1uU6nkwCQ0+mcUr1jY2P073//m1wu15Rex9jtRFGkkydPUnd3N4miKHc5c9Jkt9EH2m90Op0AgKSkJABAe3s7fD4fSktLpTbZ2dnIzMyExWIBAFgsFixbtgwGg0FqU1ZWBpfLhe7u7nt+jsfjgcvlCpqmQ6vVorS0FPX19TyIGZsWIsJ3332HGzduIDs7mzudH9C0A0gURezZswfFxcVYunQpAMBut0OtViMhISGorcFggN1ul9rcHj4TyyeW3UtlZSX0er00mUym6ZaN1NRUmEwm7g9i0zI8PIyzZ8/iqaee4n6fGTDtNVheXo4zZ85g//79M1nPPVVUVMDpdEpTX1/ftN9LEAQsXboU165dw+XLlzmE2KT5/X7U19ejuLg44h+nM1OmFUC7d+/GoUOHcPz4cWRkZEjzjUYjvF4vhoaGgto7HA4YjUapzZ1nxSZ+nmhzJ41GA51OFzQ9CIVCgdLSUpw+fRojIyMP9F4sMhAR6uvrkZ6ezo/TmUFTCiAiwu7du3Hw4EHU1tYiKysraHlubi5UKhVqamqkeT09Pejt7YXZbAYAmM1mdHV1YWBgQGpTXV0NnU43q9dSaLVaFBYWorGxkfuD2I8iIvT29sLv92PlypXc7zOTptKz/corr5Ber6cTJ06QzWaTprGxManNrl27KDMzk2pra8lqtZLZbCaz2Swt9/v9tHTpUtq8eTN1dnZSVVUVpaamUkVFxYz3sP8UURSpvb2dTp48yWcz2H0NDw/TZ599Rm63W+5SQsZkt9EpBRCAe04fffSR1MbtdtOrr75KiYmJFBMTQzt27CCbzRb0PpcvX6YtW7aQVqullJQUevPNN8nn8834l5sMv99PX3/9NfX29j7we7Hw4/V66cCBA9Tf3y93KSFlstuoQBR6vbAulwt6vR5Op/OB+4OAWyPYVVVVYfPmzYiJieFdbAbg1qFXe3u7NNAd/7uYvMluo3weEUB0dDQKCgpw7NgxHk+aSa5cuYKbN29yv89DxAGE/xs/KCMjA9988w2fmmdwuVxoaWlBcXExj+/zEHEA/UAQBKxcuRIOhwM2m03ucpiMAoEAGhsbUVRUxA8TfMg4gG4TFRWFdevWobGxMegGWxY5iAidnZ2YN28eP0xwFnAA3SEmJka6Poj7gyILEaGvrw/Xrl3D8uXLOXxmAQfQHQRBQGZmJpKTk9HV1cX9QRHE7XajpaWFx3WeRRxA9zAxnvSVK1dw9epVDqEIEAgEUF9fjzVr1iA2NlbuciIGB9B9REVFoaSkBK2trfx8sTBHRPjmm2+QlpaG9PR0ucuJKBxAP0Kv1yM/Px+NjY08nnSYIiL09/fjypUrWLFiBff7zDIOoJ+QkZGB5ORkdHR08KFYGHK73Whra8OmTZt4fB8Z8Br/CYIgYMWKFbDb7ejv7+cQCiN+vx8NDQ0oKChAbGws7/3IgANoEpRKJTZt2sT9QWGEfnieV3JyMubNmyd3ORGLA2iSJsYPOnHiBI8fFAb6+vowMjKCVatW8Z6PjDiAJmni+WKpqan8vPkQRkQYHh5GR0cHSkpK+HofmXEATcHEsAwul4ufNx+iAoEADh8+jKKiIn6O+xzAATRFE8+b7+zsxM2bN3lPKISIoojW1lYsW7YMRqORD73mAA6gadBoNCguLkZdXR33B4UIIsKFCxfg9/uRnZ0tdznsBxxA0yAIApKTk/HEE0/AYrHwTatzHBFhYGAAVquVn+M+x/BvYpoEQcCiRYsQFRWF7u5uPhSbwzweD5qbm/Hzn/8carVa7nLYbTiAHoAgCCgoKMD58+d5ELM5KhAIoKGhAatXr0Z8fLzc5bA7cAA9IJVKhaeffhptbW38kMM5ZmJQ+YSEBGRkZHCn8xzEATQDYmJiYDabUVdXB5/PJ3c5DLfC53//+x/cbjc/0WIO4wCaAYIgIDU1FQsWLEBzczN3SsuMiHDjxg2cPXsWa9as4YsN5zAOoBkiCAJycnKg0Wi4U1pmbrcbR48eRUlJCZRKpdzlsB/BATSDBEFAXl4e+vr6cPnyZQ4hGfj9ftTX12Pjxo1ITEzkQ685jgNohikUCjz55JM4deoUXyk9y0RRRFNTExYsWACj0Sh3OWwSOIAegtjYWJSWlqKxsRHj4+NylxMRiAhdXV2IiorC448/zns+IWJKAbRv3z4sX74cOp0OOp0OZrMZR44ckZaPj4+jvLwcycnJiIuLw7PPPguHwxH0Hr29vdi6dStiYmKQlpaGt956KyxvZ4iPj0dubi6qqqrg9XrlLiesEREuX76MwcFBFBUVcfiEkCkFUEZGBt577z20t7fDarVi48aNeOaZZ9Dd3Q0AeP311/HVV1/hwIEDqKurQ39/P3bu3Cm9PhAIYOvWrfB6vWhqasInn3yCjz/+GHv37p3ZbzUHTDzuecmSJWhqauIzYw8JEcFut6O7u5sfoxyK6AElJibSP//5TxoaGiKVSkUHDhyQlp07d44AkMViISKiw4cPk0KhILvdLrXZt28f6XQ68ng8k/5Mp9NJAMjpdD5o+Q+dKIrU0dFBVquVRFGUu5yw43K56IsvvqCRkRG5S2G3mew2Ou0+oEAggP3792N0dBRmsxnt7e3w+XwoLS2V2mRnZyMzMxMWiwUAYLFYsGzZMhgMBqlNWVkZXC6XtBcVbibGlB4cHMTp06e5U3oGjY2NoaGhAWvXruVneYWoKQdQV1cX4uLioNFosGvXLhw8eBA5OTmw2+1Qq9VISEgIam8wGGC32wEAdrs9KHwmlk8sux+PxwOXyxU0hRKFQoENGzZgcHCQT8/PEI/Hg6qqKqxYseKuf3MsdEw5gB5//HF0dnaipaUFr7zyCl544QWcPXv2YdQmqayshF6vlyaTyfRQP+9hUCqVKCkpwblz52C32zmEHkAgEEBjYyNWr16N+fPnc6dzCJtyAKnVaixatAi5ubmorKzEihUr8Le//Q1GoxFerxdDQ0NB7R0Oh3RNhtFovOus2MTPP3bdRkVFBZxOpzT19fVNtew5QaVSYf369bBarbhx44bc5YSkifDJyMjAI488Inc57AE98HVAoijC4/EgNzcXKpUKNTU10rKenh709vbCbDYDAMxmM7q6ujAwMCC1qa6uhk6nQ05Ozn0/Q6PRSKf+J6ZQFRMTgw0bNqCtrQ03b96Uu5yQIooirFYrtFotHnvsMd7zCQdT6dl+++23qa6uji5dukSnT5+mt99+mwRBoK+//pqIiHbt2kWZmZlUW1tLVquVzGYzmc1m6fV+v5+WLl1Kmzdvps7OTqqqqqLU1FSqqKh4KD3sc5UoijQ0NET79++nwcFBPjs2CYFAgKxWK3V2dlIgEJC7HPYTJruNTimAfvvb39IjjzxCarWaUlNTadOmTVL4EBG53W569dVXKTExkWJiYmjHjh1ks9mC3uPy5cu0ZcsW0mq1lJKSQm+++Sb5fL6plBHyATThxo0bVFVVRS6XS+5S5rRAIEBNTU108uRJDp8QMdltVCAKvd5Ql8sFvV4Pp9MZ0odjAHDz5k00NDSgpKSEb568B1EU0dXVBa/Xi9zcXB7POURMdhvl36bMEhISYDab0dTUhOHhYbnLmVNEUURzczN8Ph+HT5ji36jMJgYzKy4uRlNTEwYHB/kUPW6FT0dHBwBg9erVHD5hin+rc0RCQgIKCwtx7NixiD9FHwgE0NzcDJVKxY/RCXP8m50jBEFAYmIitmzZgra2NthstojcE/L5fKirq4NGo8Hy5cs5fMIc/3bnmLi4OGzYsAEdHR24cOFCRIXQ6OgoamtrsWDBAh5IPkJwAM0xgiAgOjoamzdvhs1mw+nTp8N+KA8igtPpxJEjR7BkyRJkZWVx+EQIPg0/hwUCAbS2tsLtdmPt2rVQqVTTeh+Px4O//OUviIqKwrx58zBv3jykpKQgPj4e8fHxiIuLQ3R0tLTR3/nnw0RE+P7773H69GmUlJQgKSnpoX8me/gmu41yAM1xRISLFy/i/PnzKCkpQXx8/JSDwWazoaCgAFevXgVwK1hUKpUUQPHx8UhKSoLRaJQCKi0tDampqUhNTUVaWhoyMjJmfLAvURTR3d2N8+fPY+vWrdBoNLznEyYmu43yM0vmOEEQsHDhQuh0OlRVVaGoqAgmk2lKG+rFixdx/fp1qT+JiODxeODxeHD9+vV7vkahUEClUkGpVCI1NRUWi2XGBnqf+PzGxkbodDo888wz/PicCMV9QCFg4lqhbdu2oaenB42NjVMaR3t4eHjKg+NP3GQ8OjoKt9s91ZLvi4gwMDCAQ4cOIT09Hfn5+Rw+EYwDKERMdE5v2rQJ0dHR+PLLL4P2an6Mx+OR/h4VFQW1Wj2lw6ns7OwZOdT1+XxoaGhAS0sLysrKsGTJEj7kinD8X0+IUSgUyM/Px6JFi1BbW4u0tDQUFRVBqVTed2O22WwQBAGLFy9GUVER9Ho9BgYG0NDQgP7+/qD31ul0UCqVcLlc0tM8UlNTodFopl0zEcFms6G1tRUZGRk8eDyTcACFqISEBDzzzDM4ffo0vvzyS+Tl5cFkMt3zwr3W1lZkZWVhx44d0tmupKQkpKWl4dNPP8X169cRExOD0tJSZGdnQ6lUwuFw4OjRo7hy5Qo0Gs20LwgcGxtDW1sb+vv7sWXLFuj1et7rYRI+BAtRgiBAqVRi1apVeOqpp9Dd3Y3Dhw/f87DM6/Vi/fr10Gq1QafYk5OTpVsdNmzYgFWrViEmJgZqtRomkwnbtm2TBoCbSmgQEXw+H6xWKz7//HPMmzcPv/rVr5CQkMDhw4LwHlCIEwQBOp0OW7ZswcDAAKqrqxEfH4/CwkKkpKRAEAS8++67+OKLLzA2NnbX641GI7Kzs7F8+fK7wiE1NRU5OTnSiJaT4fF40NXVhQsXLmDhwoXYuXMnn15n98XXAYWZQCAg9bf4/X4UFBTAaDTiq6++uufDA4qLi5GVlYVPP/0UgUDgruU2mw0vvfQScnNz7/uZRITh4WF0dXWhr68PGRkZ0t4UB09k4uuAIlRUVBTmz5+P7du349q1a+jq6kJVVRUMBgOUSiV8Pp8UCgsXLkRxcTEUCgVMJhMuX74c9F5arRbvvPMO0tPTg+ZP/J/ldrtx4cIFnD17FgqFAosXL8bOnTuhUqk4eNikcACFIUEQIAgCDAYD0tLSUFJSgt7eXoyNjcFms8Fut2PhwoVYv349gFuPDNqyZQs+++wzDAwMgIig1WqxadMmKXx8Ph88Hg8GBwfx/fff48KFC1Jf0caNG5GSksJ3rrMp40OwCEJE8Hq9GBgYwIULF3Dz5k3Y7XbExcUBuBVcDocDY2NjWLhwoXS90OjoKERRRHR0NGJjY7F48WKkpqYiISGBQ4fdEx+CsbsIggCNRgOTyQSTyQQigiiKEEURw8PDcLvd0p6OWq0GEUGn0yEuLk46rJqYGJsJHEARTBAEREVFISoqiu9CZ7Lg/WfGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsgnJWzEm7p91uVwyV8IYu5eJbfOn7nUPyQC6ceMGAMBkMslcCWPsxwwPD0Ov1993eUgG0MSNk729vT/65Vgwl8sFk8mEvr4+HsZkknidTc/EKJl3DmZ3p5AMoIkxaPR6Pf+jmIaJgebZ5PE6m7rJ7BxwJzRjTDYcQIwx2YRkAGk0Grz77rsP9LTOSMTrbep4nT1cITkmNGMsPITkHhBjLDxwADHGZMMBxBiTDQcQY0w2IRlAH3zwARYsWIDo6GgUFhaitbVV7pJkU1lZifz8fMTHxyMtLQ3bt29HT09PUJvx8XGUl5cjOTkZcXFxePbZZ+FwOILa9Pb2YuvWrYiJiUFaWhreeust+P3+2fwqsnnvvfcgCAL27NkjzeN1NksoxOzfv5/UajX961//ou7ubnrppZcoISGBHA6H3KXJoqysjD766CM6c+YMdXZ20s9+9jPKzMykkZERqc2uXbvIZDJRTU0NWa1WKioqojVr1kjL/X4/LV26lEpLS+nUqVN0+PBhSklJoYqKCjm+0qxqbW2lBQsW0PLly+m1116T5vM6mx0hF0AFBQVUXl4u/RwIBCg9PZ0qKytlrGruGBgYIABUV1dHRERDQ0OkUqnowIEDUptz584RALJYLEREdPjwYVIoFGS326U2+/btI51ORx6PZ3a/wCwaHh6mxYsXU3V1NT355JNSAPE6mz0hdQjm9XrR3t6O0tJSaZ5CoUBpaSksFouMlc0dTqcTwP/dsNve3g6fzxe0zrKzs5GZmSmtM4vFgmXLlsFgMEhtysrK4HK50N3dPYvVz67y8nJs3bo1aN0AvM5mU0jdjHr9+nUEAoGgXzoAGAwGfPvttzJVNXeIoog9e/aguLgYS5cuBQDY7Xao1WokJCQEtTUYDLDb7VKbe63TiWXhaP/+/ejo6EBbW9tdy3idzZ6QCiD248rLy3HmzBk0NjbKXcqc1tfXh9deew3V1dWIjo6Wu5yIFlKHYCkpKYiKirrrbITD4YDRaJSpqrlh9+7dOHToEI4fP46MjAxpvtFohNfrxdDQUFD729eZ0Wi85zqdWBZu2tvbMTAwgNWrV0OpVEKpVKKurg7vv/8+lEolDAYDr7NZElIBpFarkZubi5qaGmmeKIqoqamB2WyWsTL5EBF2796NgwcPora2FllZWUHLc3NzoVKpgtZZT08Pent7pXVmNpvR1dWFgYEBqU11dTV0Oh1ycnJm54vMok2bNqGrqwudnZ3SlJeXh+eff176O6+zWSJ3L/hU7d+/nzQaDX388cd09uxZevnllykhISHobEQkeeWVV0iv19OJEyfIZrNJ09jYmNRm165dlJmZSbW1tWS1WslsNpPZbJaWT5xS3rx5M3V2dlJVVRWlpqZG1Cnl28+CEfE6my0hF0BERH//+98pMzOT1Go1FRQUUHNzs9wlyQbAPaePPvpIauN2u+nVV1+lxMREiomJoR07dpDNZgt6n8uXL9OWLVtIq9VSSkoKvfnmm+Tz+Wb528jnzgDidTY7eDgOxphsQqoPiDEWXjiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJpv/Dx6bgoGgDxtbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "env.action_space= Box(-1.0, 1.0, (1,), float32)\n",
      "state= [-0.52412355  0.        ]\n",
      "action= [0.07792247]\n",
      "next_state= [-5.2400273e-01  1.2081970e-04]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ddpg.ddpg.DDPG at 0x7fd43908b3d0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import DDPG\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = DDPG(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    buffer_size=10_10000,  #replay_buff_size\n",
    "    learning_starts=100,  #从N步以后开始训练\n",
    "    batch_size=100,  #一次采样数据量\n",
    "    tau=0.005,  #target网络每次软更新量\n",
    "    gamma=0.9,\n",
    "    train_freq=(20, 'step'),  #训练频率\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-400.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/6.DDPG.Mountain Car Continuous')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-8.55, 130.48504703604928)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = DDPG.load('save/6.DDPG.Mountain Car Continuous')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhj0lEQVR4nO3de1BU9/k/8PfZ+3LZBbnsioAYxSDeBYQVL1GJxJAmmkwnzWSmmbRNRoOdmGQyE2Z+MdOZdsi0/7RpM2Y6nSb5o1NbO7VpHMUgKBdZ7hIRlYRUAgoLKLDLZVl29zy/Pwznm43GgOKevTyvmTOTPeezu88ePO+c8znnfI5ARATGGJOBQu4CGGPhiwOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYb2QLo/fffR1paGnQ6HXJzc9HY2ChXKYwxmcgSQP/4xz/w+uuv45133kFrayvWrl2LwsJCDA4OylEOY0wmghw3o+bm5iInJwd/+tOfAACiKCIlJQW//OUv8dZbb/m7HMaYTFT+/sLp6Wm0tLSgpKREmqdQKFBQUACr1XrH97hcLrhcLum1KIoYHh5GXFwcBEF44DUzxuaGiDA2NoakpCQoFN9/oOX3ALpx4wa8Xi9MJpPPfJPJhCtXrtzxPaWlpfjVr37lj/IYY/Oot7cXycnJ37vc7wF0L0pKSvD6669Lr+12O1JTU9Hb2wuDwSBjZYyxO3E4HEhJSUF0dPRd2/k9gOLj46FUKjEwMOAzf2BgAGaz+Y7v0Wq10Gq1t803GAwcQIwFsB/qIvH7WTCNRoOsrCxUVFRI80RRREVFBSwWi7/LYYzJSJZDsNdffx0vvPACsrOzsXHjRvz+97/HxMQEXnzxRTnKYYzJRJYAevbZZzE0NIRDhw7BZrNh3bp1KCsru61jmjEW2mS5Duh+ORwOGI1G2O127gNiLADNdhsNirNgjLEHb3r6OrzeEajVKVAqIwCoHvh1dhxAjDEAwOjof9DX9/+gUiVCp8tARMRa6PVroNOthFptglJpxMx5q/kKJg4gxtg3RHi9o/B6R+FyfQG7/b8AVFAodNBoUqDVLkNk5EYYDIWIjMyZl2/kAGKMfQ8FVKoYqFQLodevlPaI1OqkefsGDiDGGADA7VZBp1sHvX7FN2GzHlrtEqjVSVAodAAU894nxAHEGAMAtLYmYNOmT7BgwSIIgtIv38kjIjLGAAButwKCEOW38AE4gBhjuDV8htfrhSiKfv1ePgRjLEwREdxuN3p6elBTU4OPP/4YQ0ND2L9/v9/G2eIAYiyMEBGcTie6urpw5swZlJWVoaWlBcPDw9Ie0C9+8QtoNBq/1MMBxFgIm7nTanx8HFeuXMGpU6fw2Wef4cKFC7Db7be1b21tRVdXFzIzM/1SHwcQYyFIFEWMjIzgwoULOHXqFE6fPo0rV65gcnISd7v9c3JyEsePH8eKFSv8chjGAcRYCCEinD17Fv/85z9x5swZ9PT0wOl0zun99fX1cLvdfjkM4wBiLIQQEf785z/jyJEjs35PfHw80tLSMDo6ihs3bsBqtaK/vx+LFy9+gJXewqfhGQshgiDgiSeegFI5u2t50tPT8dOf/hRFRUXYsGEDnnvuOURHR+PcuXMPuNJbOIAYCyGCIGDr1q1ISEjwma/X6xEXF+czSLzRaMTjjz8Og8Eg9fcYDAbs3r0b7e3tfqmXD8EYCzFmsxl5eXn4z3/+AwBYuXIltm3bhpiYGDidTjQ3N6Ourg4mkwkxMTG3vT82NhbPPvusX2rlPSDGQoxarcb27dsBACkpKSgqKkJiYiI0Gg2MRiO2bduGdevWISoq6o5nupRK5W1PoSEiaRJF0ef1/eA9IMZC0I4dO2A0GpGdnQ29Xu+zTKVSYe/evdizZw/Onz+Pr776ymf50qVLER0djevXr6O/vx9DQ0OYmpqC3W6HXq/H+Pg49Ho91Go1dDodFixYgKioKCxatAjR0dFzOnvGAcRYCFq+fDnWrFkDrVZ7x72c6OhopKenIykpCadOnUJXVxdEUYTT6YTdbscnn3wCs9mMhIQELFu2DLGxsVCr1T6PWRZFEVNTUxgeHkZ/fz+uXr0Ku92OyMhIxMbGzqpODiDGQpBarcZvfvMbXLt2TQqXb1u2bBkEQcDExAR0Oh0iIyPx+OOPY8uWLUhOTvY5BLvbBYlGoxEmkwkZGRnSvNHRUXR2ds6qTn4qBmMhzOPxoKKiAi0tLXC73VAqlUhKSkJmZiYuX74Ms9mMNWvWIDExESrV/A1Cz0/FYIxBpVJh586dWL58Ofr6+tDd3Y2JiQkIgoAf//jHiIqKkrc+Wb+dMfbACYKAsbExdHd3Y8WKFVi1ahXUarXfhty4Gw4gxkIUEeH69etoampCfHw89uzZg4iIiIAInhkcQIyFoOnpaTQ2NmJgYAA7duxATExMQAXPDA4gxkIIEaGvrw9nzpxBdnY2LBbLrO8LkwMHEGMhwuv1oq2tDQMDA3jyyScRHR0dkHs93zbnWzGqq6vxox/9CElJSRAEQbrfZAYR4dChQ1i4cCH0ej0KCgrw5Zdf+rQZHh7G888/D4PBgJiYGPz85z/H+Pj4ff0QxsLZxMQETp06BUEQUFhY6HODaSCbcwBNTExg7dq1eP/99++4/Le//S3ee+89fPDBB2hoaEBkZCQKCwsxNTUltXn++efR0dGB8vJyHD9+HNXV1Xj55Zfv/VcwFqaICL29vTh9+jTWr1+P9evXB/Qh123oPgCgY8eOSa9FUSSz2Uy/+93vpHmjo6Ok1Wrp73//OxERXbp0iQBQU1OT1ObkyZMkCAJdv359Vt9rt9sJANnt9vspn7GgJooiXblyhf71r3/R+Pg4iaIod0mS2W6j83o3/NWrV2Gz2VBQUCDNMxqNyM3NhdVqBQBYrVbExMQgOztbalNQUACFQoGGhoY7fq7L5YLD4fCZGAtnHo8HVqsVQ0NDePLJJxEZGRkUh1zfNa8BZLPZAAAmk8lnvslkkpbZbDYkJib6LFepVFiwYIHU5rtKS0thNBqlKSUlZT7LZiyouN1u1NTUQK1WIz8/H2q1Wu6S7llQjAdUUlICu90uTb29vXKXxJjfERGmpqZQVVWF1NRUZGdnB+Vez7fNawCZzWYAwMDAgM/8gYEBaZnZbMbg4KDPco/Hg+HhYanNd2m1WhgMBp+JsXBCRHC5XCgvL8fDDz+Mhx56KOjDB5jnAFqyZAnMZjMqKiqkeQ6HAw0NDbBYLAAAi8WC0dFRtLS0SG0qKyshiiJyc3PnsxzGQsbExAQqKyuRl5eH5OTkkAgf4B4uRBwfH0dXV5f0+urVq2hra8OCBQuQmpqKgwcP4te//jXS09OxZMkSvP3220hKSsKePXsAACtWrMBjjz2Gl156CR988AHcbjcOHDiAn/zkJ0hKSpq3H8ZYKCAijI+Po6KiAlu3bsWCBQvkLml+zfX02pkzZwjAbdMLL7xARLdODb799ttkMplIq9XSzp07qbOz0+czbt68Sc899xxFRUWRwWCgF198kcbGxmZdA5+GZ+HC4XDQyZMnaWRkJKBOs/+Q2W6jPCAZYwGIiDA6Oopz585h8+bNd3x6RSCb7TYaFGfBGAsnRITJyUmcO3cOFosFRqNR7pIeGL4ZlbEAMzU1hbNnz2LTpk2h1+fzHbwHxFgAcblcqKioQHZ29qyfLBHMOIAYCxAzVzivW7cOiYmJIXOq/W44gBgLAG63GxUVFXjooYewaNGisAgfgAOIMdkRERobG2E2m7FkyZKwCR+AA4gxWRER2tvbodFosHbt2rAKH4ADiDHZEBEuXryIvr4+ZGVlhV34ABxAjMmmv78ffX192LFjh88z18NJeP5qxmQ2c0P21q1bodFo5C5HNhxAjPmZ0+nEiRMnYLFYoNPp5C5HVkF9JXQQ3sbGwpzX60V1dTW2bduG+Ph4ucuRXVDvATkcDg4hFjREUYTVakVycjIPPfONoA6guro6TE9Py10GYz+IiHD58mVMT08jMzMzLM943UlQB1BGRgYqKirg8XjkLoWx70VEGBgYQG9vL7Zs2cLh8y1BHUBpaWmIj49HW1sbH4qxgDU+Pg6r1YqtW7cG9RMsHoSgDiBBEJCVlQWHw4Gvv/6aQ4gFHLfbjbNnzyInJwd6vV7ucgJOUAcQACiVSmzevBnNzc0YGRmRuxzGJKIoor6+HsuXLw+rG0znIugDCADUajUeeeQR1NTUwOVyyV0OY9JtFgqFAunp6Rw+3yMkAkgQBMTFxWH16tWoq6uDKIpyl8TCGBFhaGgIV69eRW5ubtjeZjEbIbNmBEHAkiVLEBkZifPnz3N/EJON0+lEfX09duzYAZUqqK/1feBCJoCAWyG0YcMG2Gw22Gw2DiHmdx6PB5WVldiwYQOio6PlLifghVQAAYBKpcL27dvR0NAAp9MpdzksjBARmpubkZCQgEWLFsldTlAIuQACAL1ej5ycHJw+fRput1vuclgYICJcu3YNY2NjyM7O5k7nWQrJABIEAUlJSUhKSuL+IOYXDocDjY2N2LJlC5RKpdzlBI2QDCDgVgitX78eY2Nj6OnpkbscFsI8Hg8qKiqQm5sLrVYrdzlBJWQDCLh1kWJ+fj6am5sxPDzMe0Js3s30+yxfvhzJycl86DVHIR1AAKDVamGxWFBVVQWv1yt3OSzEfPXVV5icnERmZqbcpQSlOQVQaWkpcnJyEB0djcTEROzZswednZ0+baamplBcXIy4uDhERUXhmWeewcDAgE+bnp4eFBUVISIiAomJiXjzzTcf2B3tgiBg4cKFSE9PR3NzM+8FsXkzOjqKxsZG5OXl8cWG92hOa62qqgrFxcWor69HeXk53G43du3ahYmJCanNa6+9hk8//RRHjx5FVVUV+vr68PTTT0vLvV4vioqKMD09jbq6Onz88cf46KOPcOjQofn7Vd8hCAJWrlyJ8fFxfPXVVxxC7L653W7U1tbi0UcfRUREhNzlBC+6D4ODgwSAqqqqiIhodHSU1Go1HT16VGpz+fJlAkBWq5WIiE6cOEEKhYJsNpvU5vDhw2QwGMjlcs3qe+12OwEgu90+p3onJyfpb3/7Gzkcjjm9j7FvE0WRzp07Rx0dHSSKotzlBKTZbqP3td9ot9sBAAsWLAAAtLS0wO12o6CgQGqTkZGB1NRUWK1WAIDVasXq1athMpmkNoWFhXA4HOjo6Ljj97hcLjgcDp/pXuj1ehQUFKC6upoHMWP3hIjw5Zdf4ubNm8jIyOBO5/t0zwEkiiIOHjyI/Px8rFq1CgBgs9mg0WgQExPj09ZkMsFms0ltvh0+M8tnlt1JaWkpjEajNKWkpNxr2UhISEBKSgr3B7F7MjY2hkuXLuHRRx/lfp95cM9rsLi4GBcvXsSRI0fms547Kikpgd1ul6be3t57/ixBELBq1SoMDQ2hu7ubQ4jNmsfjQXV1NfLz88P+cTrz5Z4C6MCBAzh+/DjOnDmD5ORkab7ZbMb09DRGR0d92g8MDMBsNkttvntWbOb1TJvv0mq1MBgMPtP9UCgUKCgowIULFzA+Pn5fn8XCAxGhuroaSUlJ/DideTSnACIiHDhwAMeOHUNlZSWWLFniszwrKwtqtRoVFRXSvM7OTvT09MBisQAALBYL2tvbMTg4KLUpLy+HwWDw67UUer0eubm5qK2t5f4gdldEhJ6eHng8Hqxbt477febTXHq29+/fT0ajkc6ePUv9/f3SNDk5KbXZt28fpaamUmVlJTU3N5PFYiGLxSIt93g8tGrVKtq1axe1tbVRWVkZJSQkUElJybz3sP8QURSppaWFzp07x2cz2PcaGxujf//73+R0OuUuJWjMdhudUwABuOP04YcfSm2cTie98sorFBsbSxEREbR3717q7+/3+Zzu7m7avXs36fV6io+PpzfeeIPcbve8/7jZ8Hg89Nlnn1FPT899fxYLPdPT03T06FHq6+uTu5SgMtttVCAKvl5Yh8MBo9EIu91+3/1BwK0R7MrKyrBr1y5ERETwLjYDcOvQq6WlRRrojv9dzN5st1E+jwhAp9Nh48aNOH36NI8nzSTXrl3DyMgI9/s8QBxA+L/xg5KTk/H555/zqXkGh8OBhoYG5Ofn8/g+DxAH0DcEQcC6deswMDCA/v5+ucthMvJ6vaitrUVeXh4/TPAB4wD6FqVSia1bt6K2ttbnBlsWPogIbW1tWLhwIT9M0A84gL4jIiJCuj6I+4PCCxGht7cXQ0NDWLNmDYePH3AAfYcgCEhNTUVcXBza29u5PyiMOJ1ONDQ08LjOfsQBdAcz40lfu3YN169f5xAKA16vF9XV1di0aRMiIyPlLidscAB9D6VSic2bN6OxsZGfLxbiiAiff/45EhMTkZSUJHc5YYUD6C6MRiNycnJQW1vL40mHKCJCX18frl27hrVr13K/j59xAP2A5ORkxMXFobW1lQ/FQpDT6URTUxN27tzJ4/vIgNf4DxAEAWvXroXNZkNfXx+HUAjxeDyoqanBxo0bERkZyXs/MuAAmgWVSoWdO3dyf1AIoW+e5xUXF4eFCxfKXU7Y4gCapZnxg86ePcvjB4WA3t5ejI+PY/369bznIyMOoFmaeb5YQkICP28+iBERxsbG0Nrais2bN/P1PjLjAJqDmWEZHA4HP28+SHm9Xpw4cQJ5eXn8HPcAwAE0RzPPm29ra8PIyAjvCQURURTR2NiI1atXw2w286FXAOAAugdarRb5+fmoqqri/qAgQUTo6uqCx+NBRkaG3OWwb3AA3QNBEBAXF4eVK1fCarXyTasBjogwODiI5uZmfo57gOG/xD0SBAHLli2DUqlER0cHH4oFMJfLhfr6ejzxxBPQaDRyl8O+hQPoPgiCgI0bN+KLL77gQcwClNfrRU1NDTZs2IDo6Gi5y2HfwQF0n9RqNR577DE0NTXxQw4DzMyg8jExMUhOTuZO5wDEATQPIiIiYLFYUFVVBbfbLXc5DLfC53//+x+cTic/0SKAcQDNA0EQkJCQgLS0NNTX13OntMyICDdv3sSlS5ewadMmvtgwgHEAzRNBEJCZmQmtVsud0jJzOp04deoUNm/eDJVKJXc57C44gOaRIAjIzs5Gb28vuru7OYRk4PF4UF1djR07diA2NpYPvQIcB9A8UygU2LZtG86fP89XSvuZKIqoq6tDWloazGaz3OWwWeAAegAiIyNRUFCA2tpaTE1NyV1OWCAitLe3Q6lU4uGHH+Y9nyAxpwA6fPgw1qxZA4PBAIPBAIvFgpMnT0rLp6amUFxcjLi4OERFReGZZ57BwMCAz2f09PSgqKgIERERSExMxJtvvhmStzNER0cjKysLZWVlmJ6elruckEZE6O7uxvDwMPLy8jh8gsicAig5ORnvvvsuWlpa0NzcjB07duCpp55CR0cHAOC1117Dp59+iqNHj6Kqqgp9fX14+umnpfd7vV4UFRVhenoadXV1+Pjjj/HRRx/h0KFD8/urAsDM455XrFiBuro6PjP2gBARbDYbOjo6+DHKwYjuU2xsLP3lL3+h0dFRUqvVdPToUWnZ5cuXCQBZrVYiIjpx4gQpFAqy2WxSm8OHD5PBYCCXyzXr77Tb7QSA7Hb7/Zb/wImiSK2trdTc3EyiKMpdTshxOBz0ySef0Pj4uNylsG+Z7TZ6z31AXq8XR44cwcTEBCwWC1paWuB2u1FQUCC1ycjIQGpqKqxWKwDAarVi9erVMJlMUpvCwkI4HA5pLyrUzIwpPTw8jAsXLnCn9DyanJxETU0NtmzZws/yClJzDqD29nZERUVBq9Vi3759OHbsGDIzM2Gz2aDRaBATE+PT3mQywWazAQBsNptP+Mwsn1n2fVwuFxwOh88UTBQKBbZv347h4WE+PT9PXC4XysrKsHbt2tv+zbHgMecAevjhh9HW1oaGhgbs378fL7zwAi5duvQgapOUlpbCaDRKU0pKygP9vgdBpVJh8+bNuHz5Mmw2G4fQffB6vaitrcWGDRuwaNEi7nQOYnMOII1Gg2XLliErKwulpaVYu3Yt/vCHP8BsNmN6ehqjo6M+7QcGBqRrMsxm821nxWZe3+26jZKSEtjtdmnq7e2da9kBQa1W45FHHkFzczNu3rwpdzlBaSZ8kpOTsXjxYrnLYffpvq8DEkURLpcLWVlZUKvVqKiokJZ1dnaip6cHFosFAGCxWNDe3o7BwUGpTXl5OQwGAzIzM7/3O7RarXTqf2YKVhEREdi+fTuampowMjIidzlBRRRFNDc3Q6/XY/ny5bznEwrm0rP91ltvUVVVFV29epUuXLhAb731FgmCQJ999hkREe3bt49SU1OpsrKSmpubyWKxkMVikd7v8Xho1apVtGvXLmpra6OysjJKSEigkpKSB9LDHqhEUaTR0VE6cuQIDQ8P89mxWfB6vdTc3ExtbW3k9XrlLof9gNluo3MKoJ/97Ge0ePFi0mg0lJCQQDt37pTCh4jI6XTSK6+8QrGxsRQREUF79+6l/v5+n8/o7u6m3bt3k16vp/j4eHrjjTfI7XbPpYygD6AZN2/epLKyMnI4HHKXEtC8Xi/V1dXRuXPnOHyCxGy3UYEo+HpDHQ4HjEYj7HZ7UB+OAcDIyAhqamqwefNmvnnyDkRRRHt7O6anp5GVlcXjOQeJ2W6j/NeUWUxMDCwWC+rq6jA2NiZ3OQFFFEXU19fD7XZz+IQo/ovKbGYws/z8fNTV1WF4eJhP0eNW+LS2tgIANmzYwOETovivGiBiYmKQm5uL06dPh/0peq/Xi/r6eqjVan6MTojjv2yAEAQBsbGx2L17N5qamtDf3x+We0JutxtVVVXQarVYs2YNh0+I479ugImKisL27dvR2tqKrq6usAqhiYkJVFZWIi0tjQeSDxMcQAFGEATodDrs2rUL/f39uHDhQsgP5UFEsNvtOHnyJFasWIElS5Zw+IQJDqAApVarkZ+fj8nJSZw9ezZkH/dDRPj6669RVVWFHTt2IDU1lcMnjHAABTClUom8vDykpqaivLwcDocjpA7JRFHExYsX0dLSgl27diE2NlbukpifcQAFOEEQsHTpUmRnZ6OsrAy9vb1BH0JEhKmpKVRWVsLpdOKpp56CTqfjPZ8wxAEUBGauFXryySfR2dmJ2traoB1Hm4gwODiI48ePIykpCTk5OfzsrjDGARQkZjqnd+7cCZ1Oh//+97+4ceNGUO0Nud1u1NTUoKGhAYWFhVixYgXv9YQ5/l9PkFEoFMjJycGyZctQWVmJxMRE5OXlQaVSBezGTETo7+9HY2MjkpOTefB4JuGbUYMUEcHr9eLChQu4evUqsrOzkZKSEnAX7k1OTqKpqQl9fX3YvXs3jEZjwAYlmz+z3UY5gIIcEWFsbAy1tbUQRRF5eXmIi4uTdSMnIng8Hnz++ef44osvkJ2djaVLl/JeTxjhAAozM527lZWViI6ORm5uLuLj4/0eRC6XC+3t7ejq6sLSpUuxevVqaLVa3usJMxxAYcrr9Ur9LR6PBxs3bsSiRYseaB/RzF5Ye3s7ent7kZycjPXr1yMiIoKDJ0xxAIUxujXSJYaGhqS9kRUrVmD58uVITEyEQqG4r2CY+SfjdDrR1dWFS5cuQaFQID09HStXroRarebgCXMcQAzArbBwuVzo6elBW1sbnE4n4uPjkZSUhMWLF0Ov10uHSN8XGjN9Oi6XC8PDw/j666/R1dUFjUaDlJQUZGRkID4+PuA6wJl8OIDYbYgI09PTGBwcRFdXF0ZGRmCz2RAVFQVBEBAdHQ2dTgePxwOdTofR0VEoFApMTExAFEXodDpERkYiPT0dCQkJiImJ4dBhdzTbbZSvAwojgiBAq9UiJSUFKSkpICKIoghRFDE2Ngan0wm32w2XywWNRgMigsFgQFRUlHRYdbc9JcbmigMojAmCAKVSCaVSiQULFshdDgtDvP/MGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDZBeSvGzP2zDodD5koYY3cys23+0L3uQRlAN2/eBACkpKTIXAlj7G7GxsZgNBq/d3lQBtDMjZM9PT13/XHMl8PhQEpKCnp7e3kYk1nidXZvZkbJTEpKumu7oAygmTFojEYj/6O4BwaDgdfbHPE6m7vZ7BxwJzRjTDYcQIwx2QRlAGm1WrzzzjvQarVylxJUeL3NHa+zBysox4RmjIWGoNwDYoyFBg4gxphsOIAYY7LhAGKMySYoA+j9999HWloadDodcnNz0djYKHdJsiktLUVOTg6io6ORmJiIPXv2oLOz06fN1NQUiouLERcXh6ioKDzzzDMYGBjwadPT04OioiJEREQgMTERb775Jjwejz9/imzeffddCIKAgwcPSvN4nfkJBZkjR46QRqOhv/71r9TR0UEvvfQSxcTE0MDAgNylyaKwsJA+/PBDunjxIrW1tdHjjz9OqampND4+LrXZt28fpaSkUEVFBTU3N1NeXh5t2rRJWu7xeGjVqlVUUFBA58+fpxMnTlB8fDyVlJTI8ZP8qrGxkdLS0mjNmjX06quvSvN5nflH0AXQxo0bqbi4WHrt9XopKSmJSktLZawqcAwODhIAqqqqIiKi0dFRUqvVdPToUanN5cuXCQBZrVYiIjpx4gQpFAqy2WxSm8OHD5PBYCCXy+XfH+BHY2NjlJ6eTuXl5bRt2zYpgHid+U9QHYJNT0+jpaUFBQUF0jyFQoGCggJYrVYZKwscdrsdwP/dsNvS0gK32+2zzjIyMpCamiqtM6vVitWrV8NkMkltCgsL4XA40NHR4cfq/au4uBhFRUU+6wbgdeZPQXUz6o0bN+D1en3+6ABgMplw5coVmaoKHKIo4uDBg8jPz8eqVasAADabDRqNBjExMT5tTSYTbDab1OZO63RmWSg6cuQIWltb0dTUdNsyXmf+E1QBxO6uuLgYFy9eRG1trdylBLTe3l68+uqrKC8vh06nk7ucsBZUh2Dx8fFQKpW3nY0YGBiA2WyWqarAcODAARw/fhxnzpxBcnKyNN9sNmN6ehqjo6M+7b+9zsxm8x3X6cyyUNPS0oLBwUFs2LABKpUKKpUKVVVVeO+996BSqWAymXid+UlQBZBGo0FWVhYqKiqkeaIooqKiAhaLRcbK5ENEOHDgAI4dO4bKykosWbLEZ3lWVhbUarXPOuvs7ERPT4+0ziwWC9rb2zE4OCi1KS8vh8FgQGZmpn9+iB/t3LkT7e3taGtrk6bs7Gw8//zz0n/zOvMTuXvB5+rIkSOk1Wrpo48+okuXLtHLL79MMTExPmcjwsn+/fvJaDTS2bNnqb+/X5omJyelNvv27aPU1FSqrKyk5uZmslgsZLFYpOUzp5R37dpFbW1tVFZWRgkJCWF1SvnbZ8GIeJ35S9AFEBHRH//4R0pNTSWNRkMbN26k+vp6uUuSDYA7Th9++KHUxul00iuvvEKxsbEUERFBe/fupf7+fp/P6e7upt27d5Ner6f4+Hh64403yO12+/nXyOe7AcTrzD94OA7GmGyCqg+IMRZaOIAYY7LhAGKMyYYDiDEmGw4gxphsOIAYY7LhAGKMyYYDiDEmGw4gxphsOIAYY7LhAGKMyYYDiDEmm/8PvCAK3IlAJGUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34.0 67 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
